{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set tf 1.x for colab\n",
    "%tensorflow_version 1.x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-03T21:05:26.096298Z",
     "start_time": "2017-09-03T21:05:26.092498Z"
    }
   },
   "source": [
    "# Your first CNN on CIFAR-10\n",
    "\n",
    "In this task you will: \n",
    "* define your first CNN architecture for CIFAR-10 dataset\n",
    "* train it from scratch\n",
    "* visualize learnt filters\n",
    "\n",
    "CIFAR-10 dataset contains 32x32 color images from 10 classes: __airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck__:\n",
    "<img src=\"images/cifar10.jpg\" style=\"width:80%\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-29T18:51:38.014629Z",
     "start_time": "2017-10-29T18:51:37.876670Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"..\")\n",
    "import grading\n",
    "import download_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !!! remember to clear session/graph if you rebuild your graph to avoid out-of-memory errors !!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-29T18:51:39.175876Z",
     "start_time": "2017-10-29T18:51:39.170539Z"
    }
   },
   "outputs": [],
   "source": [
    "download_utils.link_all_keras_resources()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-29T18:51:58.907479Z",
     "start_time": "2017-10-29T18:51:42.206537Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras import backend as K\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "print(tf.__version__)\n",
    "print(keras.__version__)\n",
    "import grading_utils\n",
    "import keras_utils\n",
    "from keras_utils import reset_tf_session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fill in your Coursera token and email\n",
    "To successfully submit your answers to our grader, please fill in your Coursera submission token and email"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grader = grading.Grader(assignment_key=\"s1B1I5DuEeeyLAqI7dCYkg\", \n",
    "                        all_parts=[\"7W4tu\", \"nQOsg\", \"96eco\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# token expires every 30 min\n",
    "COURSERA_TOKEN = ### YOUR TOKEN HERE\n",
    "COURSERA_EMAIL = ### YOUR EMAIL HERE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-03T23:44:40.870302Z",
     "start_time": "2017-09-03T23:44:39.221603Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.datasets import cifar10\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-04T00:27:41.648291Z",
     "start_time": "2017-09-04T00:27:41.644322Z"
    }
   },
   "outputs": [],
   "source": [
    "print(\"Train samples:\", x_train.shape, y_train.shape)\n",
    "print(\"Test samples:\", x_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-03T23:44:41.009639Z",
     "start_time": "2017-09-03T23:44:40.877013Z"
    }
   },
   "outputs": [],
   "source": [
    "NUM_CLASSES = 10\n",
    "cifar10_classes = [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \n",
    "                   \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-03T23:44:42.285830Z",
     "start_time": "2017-09-03T23:44:41.011216Z"
    }
   },
   "outputs": [],
   "source": [
    "# show random images from train\n",
    "cols = 8\n",
    "rows = 2\n",
    "fig = plt.figure(figsize=(2 * cols - 1, 2.5 * rows - 1))\n",
    "for i in range(cols):\n",
    "    for j in range(rows):\n",
    "        random_index = np.random.randint(0, len(y_train))\n",
    "        ax = fig.add_subplot(rows, cols, i * rows + j + 1)\n",
    "        ax.grid('off')\n",
    "        ax.axis('off')\n",
    "        ax.imshow(x_train[random_index, :])\n",
    "        ax.set_title(cifar10_classes[y_train[random_index, 0]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to normalize inputs like this: $$x_{norm} = \\frac{x}{255} - 0.5$$\n",
    "\n",
    "We need to convert class labels to one-hot encoded vectors. Use __keras.utils.to_categorical__."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-04T00:25:55.504781Z",
     "start_time": "2017-09-04T00:25:55.500823Z"
    }
   },
   "outputs": [],
   "source": [
    "# normalize inputs\n",
    "x_train2 = ### YOUR CODE HERE\n",
    "x_test2 = ### YOUR CODE HERE\n",
    "\n",
    "# convert class labels to one-hot encoded, should have shape (?, NUM_CLASSES)\n",
    "y_train2 = ### YOUR CODE HERE\n",
    "y_test2 = ### YOUR CODE HERE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define CNN architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T11:39:23.631230Z",
     "start_time": "2017-08-24T11:39:23.627975Z"
    }
   },
   "outputs": [],
   "source": [
    "# import necessary building blocks\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Activation, Dropout\n",
    "from keras.layers.advanced_activations import LeakyReLU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convolutional networks are built from several types of layers:\n",
    "- [Conv2D](https://keras.io/layers/convolutional/#conv2d) - performs convolution:\n",
    "    - **filters**: number of output channels; \n",
    "    - **kernel_size**: an integer or tuple/list of 2 integers, specifying the width and height of the 2D convolution window;\n",
    "    - **padding**: padding=\"same\" adds zero padding to the input, so that the output has the same width and height, padding='valid' performs convolution only in locations where kernel and the input fully overlap;\n",
    "    - **activation**: \"relu\", \"tanh\", etc.\n",
    "    - **input_shape**: shape of input.\n",
    "- [MaxPooling2D](https://keras.io/layers/pooling/#maxpooling2d) - performs 2D max pooling.\n",
    "- [Flatten](https://keras.io/layers/core/#flatten) - flattens the input, does not affect the batch size.\n",
    "- [Dense](https://keras.io/layers/core/#dense) - fully-connected layer.\n",
    "- [Activation](https://keras.io/layers/core/#activation) - applies an activation function.\n",
    "- [LeakyReLU](https://keras.io/layers/advanced-activations/#leakyrelu) - applies leaky relu activation.\n",
    "- [Dropout](https://keras.io/layers/core/#dropout) - applies dropout."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-03T23:48:59.567910Z",
     "start_time": "2017-09-03T23:48:59.564449Z"
    }
   },
   "source": [
    "You need to define a model which takes __(None, 32, 32, 3)__ input and predicts __(None, 10)__ output with probabilities for all classes. __None__ in shapes stands for batch dimension.\n",
    "\n",
    "Simple feed-forward networks in Keras can be defined in the following way:\n",
    "\n",
    "```python\n",
    "model = Sequential()  # start feed-forward model definition\n",
    "model.add(Conv2D(..., input_shape=(32, 32, 3)))  # first layer needs to define \"input_shape\"\n",
    "\n",
    "...  # here comes a bunch of convolutional, pooling and dropout layers\n",
    "\n",
    "model.add(Dense(NUM_CLASSES))  # the last layer with neuron for each class\n",
    "model.add(Activation(\"softmax\"))  # output probabilities\n",
    "```\n",
    "\n",
    "Stack __4__ convolutional layers with kernel size __(3, 3)__ with growing number of filters __(16, 32, 32, 64)__, use \"same\" padding.\n",
    "\n",
    "Add __2x2__ pooling layer after every 2 convolutional layers (conv-conv-pool scheme).\n",
    "\n",
    "Use __LeakyReLU__ activation with recommended parameter __0.1__ for all layers that need it (after convolutional and dense layers):\n",
    "```python\n",
    "model.add(LeakyReLU(0.1))\n",
    "```\n",
    "\n",
    "Add a dense layer with __256__ neurons and a second dense layer with __10__ neurons for classes. Remember to use __Flatten__ layer before first dense layer to reshape input volume into a flat vector!\n",
    "\n",
    "Add __Dropout__ after every pooling layer (__0.25__) and between dense layers (__0.5__)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T11:39:23.739649Z",
     "start_time": "2017-08-24T11:39:23.632558Z"
    }
   },
   "outputs": [],
   "source": [
    "def make_model():\n",
    "    \"\"\"\n",
    "    Define your model architecture here.\n",
    "    Returns `Sequential` model.\n",
    "    \"\"\"\n",
    "    model = Sequential()\n",
    "\n",
    "    ### YOUR CODE HERE\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T11:39:23.948546Z",
     "start_time": "2017-08-24T11:39:23.741012Z"
    }
   },
   "outputs": [],
   "source": [
    "# describe model\n",
    "s = reset_tf_session()  # clear default graph\n",
    "model = make_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## GRADED PART, DO NOT CHANGE!\n",
    "# Number of model parameters\n",
    "grader.set_answer(\"7W4tu\", grading_utils.model_total_params(model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you can make submission with answers so far to check yourself at this stage\n",
    "grader.submit(COURSERA_EMAIL, COURSERA_TOKEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training of your model can take approx. 4-8 minutes per epoch.\n",
    "\n",
    "During training you should observe the decrease in reported loss on training and validation.\n",
    "\n",
    "If the loss on training is not decreasing with epochs you should revise your model definition and learning rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T12:18:39.059726Z",
     "start_time": "2017-08-24T11:39:23.949926Z"
    }
   },
   "outputs": [],
   "source": [
    "INIT_LR = 5e-3  # initial learning rate\n",
    "BATCH_SIZE = 32\n",
    "EPOCHS = 10\n",
    "\n",
    "s = reset_tf_session()  # clear default graph\n",
    "# don't call K.set_learning_phase() !!! (otherwise will enable dropout in train/test simultaneously)\n",
    "model = make_model()  # define our model\n",
    "\n",
    "# prepare model for fitting (loss, optimizer, etc)\n",
    "model.compile(\n",
    "    loss='categorical_crossentropy',  # we train 10-way classification\n",
    "    optimizer=keras.optimizers.adamax(lr=INIT_LR),  # for SGD\n",
    "    metrics=['accuracy']  # report accuracy during training\n",
    ")\n",
    "\n",
    "# scheduler of learning rate (decay with epochs)\n",
    "def lr_scheduler(epoch):\n",
    "    return INIT_LR * 0.9 ** epoch\n",
    "\n",
    "# callback for printing of actual learning rate used by optimizer\n",
    "class LrHistory(keras.callbacks.Callback):\n",
    "    def on_epoch_begin(self, epoch, logs={}):\n",
    "        print(\"Learning rate:\", K.get_value(model.optimizer.lr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training takes approximately **1.5 hours**. You're aiming for ~0.80 validation accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we will save model checkpoints to continue training in case of kernel death\n",
    "model_filename = 'cifar.{0:03d}.hdf5'\n",
    "last_finished_epoch = None\n",
    "\n",
    "#### uncomment below to continue training from model checkpoint\n",
    "#### fill `last_finished_epoch` with your latest finished epoch\n",
    "# from keras.models import load_model\n",
    "# s = reset_tf_session()\n",
    "# last_finished_epoch = 7\n",
    "# model = load_model(model_filename.format(last_finished_epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T12:18:39.059726Z",
     "start_time": "2017-08-24T11:39:23.949926Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# fit model\n",
    "model.fit(\n",
    "    x_train2, y_train2,  # prepared data\n",
    "    batch_size=BATCH_SIZE,\n",
    "    epochs=EPOCHS,\n",
    "    callbacks=[keras.callbacks.LearningRateScheduler(lr_scheduler), \n",
    "               LrHistory(), \n",
    "               keras_utils.TqdmProgressCallback(),\n",
    "               keras_utils.ModelSaveCallback(model_filename)],\n",
    "    validation_data=(x_test2, y_test2),\n",
    "    shuffle=True,\n",
    "    verbose=0,\n",
    "    initial_epoch=last_finished_epoch or 0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T12:18:39.103672Z",
     "start_time": "2017-08-24T12:18:39.061508Z"
    }
   },
   "outputs": [],
   "source": [
    "# save weights to file\n",
    "model.save_weights(\"weights.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T12:18:39.298255Z",
     "start_time": "2017-08-24T12:18:39.105314Z"
    }
   },
   "outputs": [],
   "source": [
    "# load weights from file (can call without model.fit)\n",
    "model.load_weights(\"weights.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T12:18:53.381943Z",
     "start_time": "2017-08-24T12:18:39.299830Z"
    }
   },
   "outputs": [],
   "source": [
    "# make test predictions\n",
    "y_pred_test = model.predict_proba(x_test2)\n",
    "y_pred_test_classes = np.argmax(y_pred_test, axis=1)\n",
    "y_pred_test_max_probas = np.max(y_pred_test, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T12:18:54.293970Z",
     "start_time": "2017-08-24T12:18:53.383809Z"
    }
   },
   "outputs": [],
   "source": [
    "# confusion matrix and accuracy\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
    "plt.figure(figsize=(7, 6))\n",
    "plt.title('Confusion matrix', fontsize=16)\n",
    "plt.imshow(confusion_matrix(y_test, y_pred_test_classes))\n",
    "plt.xticks(np.arange(10), cifar10_classes, rotation=45, fontsize=12)\n",
    "plt.yticks(np.arange(10), cifar10_classes, fontsize=12)\n",
    "plt.colorbar()\n",
    "plt.show()\n",
    "print(\"Test accuracy:\", accuracy_score(y_test, y_pred_test_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## GRADED PART, DO NOT CHANGE!\n",
    "# Accuracy on validation data\n",
    "grader.set_answer(\"nQOsg\", accuracy_score(y_test, y_pred_test_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you can make submission with answers so far to check yourself at this stage\n",
    "grader.submit(COURSERA_EMAIL, COURSERA_TOKEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T12:18:55.568152Z",
     "start_time": "2017-08-24T12:18:54.295958Z"
    }
   },
   "outputs": [],
   "source": [
    "# inspect preditions\n",
    "cols = 8\n",
    "rows = 2\n",
    "fig = plt.figure(figsize=(2 * cols - 1, 3 * rows - 1))\n",
    "for i in range(cols):\n",
    "    for j in range(rows):\n",
    "        random_index = np.random.randint(0, len(y_test))\n",
    "        ax = fig.add_subplot(rows, cols, i * rows + j + 1)\n",
    "        ax.grid('off')\n",
    "        ax.axis('off')\n",
    "        ax.imshow(x_test[random_index, :])\n",
    "        pred_label = cifar10_classes[y_pred_test_classes[random_index]]\n",
    "        pred_proba = y_pred_test_max_probas[random_index]\n",
    "        true_label = cifar10_classes[y_test[random_index, 0]]\n",
    "        ax.set_title(\"pred: {}\\nscore: {:.3}\\ntrue: {}\".format(\n",
    "               pred_label, pred_proba, true_label\n",
    "        ))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize maximum stimuli"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to find input images that provide maximum activations for particular layers of our network. \n",
    "\n",
    "We will find those maximum stimuli via gradient ascent in image space.\n",
    "\n",
    "For that task we load our model weights, calculate the layer output gradient with respect to image input and shift input image in that direction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T13:23:59.462081Z",
     "start_time": "2017-08-24T13:23:58.896876Z"
    }
   },
   "outputs": [],
   "source": [
    "s = reset_tf_session()  # clear default graph\n",
    "K.set_learning_phase(0)  # disable dropout\n",
    "model = make_model()\n",
    "model.load_weights(\"weights.h5\")  # that were saved after model.fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T13:24:00.360163Z",
     "start_time": "2017-08-24T13:24:00.351539Z"
    }
   },
   "outputs": [],
   "source": [
    "# all weights we have\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-09T19:26:28.647877Z",
     "start_time": "2017-09-09T19:26:28.587968Z"
    }
   },
   "outputs": [],
   "source": [
    "def find_maximum_stimuli(layer_name, is_conv, filter_index, model, iterations=20, step=1., verbose=True):\n",
    "    \n",
    "    def image_values_to_rgb(x):\n",
    "        # normalize x: center on 0 (np.mean(x_train2)), ensure std is 0.25 (np.std(x_train2))\n",
    "        # so that it looks like a normalized image input for our network\n",
    "        x = ### YOUR CODE HERE\n",
    "\n",
    "        # do reverse normalization to RGB values: x = (x_norm + 0.5) * 255\n",
    "        x = ### YOUR CODE HERE\n",
    "    \n",
    "        # clip values to [0, 255] and convert to bytes\n",
    "        x = np.clip(x, 0, 255).astype('uint8')\n",
    "        return x\n",
    "\n",
    "    # this is the placeholder for the input image\n",
    "    input_img = model.input\n",
    "    img_width, img_height = input_img.shape.as_list()[1:3]\n",
    "    \n",
    "    # find the layer output by name\n",
    "    layer_output = list(filter(lambda x: x.name == layer_name, model.layers))[0].output\n",
    "\n",
    "    # we build a loss function that maximizes the activation\n",
    "    # of the filter_index filter of the layer considered\n",
    "    if is_conv:\n",
    "        # mean over feature map values for convolutional layer\n",
    "        loss = K.mean(layer_output[:, :, :, filter_index])\n",
    "    else:\n",
    "        loss = K.mean(layer_output[:, filter_index])\n",
    "\n",
    "    # we compute the gradient of the loss wrt input image\n",
    "    grads = K.gradients(loss, input_img)[0]  # [0] because of the batch dimension!\n",
    "\n",
    "    # normalization trick: we normalize the gradient\n",
    "    grads = grads / (K.sqrt(K.sum(K.square(grads))) + 1e-10)\n",
    "\n",
    "    # this function returns the loss and grads given the input picture\n",
    "    iterate = K.function([input_img], [loss, grads])\n",
    "\n",
    "    # we start from a gray image with some random noise\n",
    "    input_img_data = np.random.random((1, img_width, img_height, 3))\n",
    "    input_img_data = (input_img_data - 0.5) * (0.1 if is_conv else 0.001)\n",
    "\n",
    "    # we run gradient ascent\n",
    "    for i in range(iterations):\n",
    "        loss_value, grads_value = iterate([input_img_data])\n",
    "        input_img_data += grads_value * step\n",
    "        if verbose:\n",
    "            print('Current loss value:', loss_value)\n",
    "\n",
    "    # decode the resulting input image\n",
    "    img = image_values_to_rgb(input_img_data[0])\n",
    "    \n",
    "    return img, loss_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T13:32:09.846317Z",
     "start_time": "2017-08-24T13:32:09.815734Z"
    }
   },
   "outputs": [],
   "source": [
    "# sample maximum stimuli\n",
    "def plot_filters_stimuli(layer_name, is_conv, model, iterations=20, step=1., verbose=False):\n",
    "    cols = 8\n",
    "    rows = 2\n",
    "    filter_index = 0\n",
    "    max_filter_index = list(filter(lambda x: x.name == layer_name, model.layers))[0].output.shape.as_list()[-1] - 1\n",
    "    fig = plt.figure(figsize=(2 * cols - 1, 3 * rows - 1))\n",
    "    for i in range(cols):\n",
    "        for j in range(rows):\n",
    "            if filter_index <= max_filter_index:\n",
    "                ax = fig.add_subplot(rows, cols, i * rows + j + 1)\n",
    "                ax.grid('off')\n",
    "                ax.axis('off')\n",
    "                loss = -1e20\n",
    "                while loss < 0 and filter_index <= max_filter_index:\n",
    "                    stimuli, loss = find_maximum_stimuli(layer_name, is_conv, filter_index, model,\n",
    "                                                         iterations, step, verbose=verbose)\n",
    "                    filter_index += 1\n",
    "                if loss > 0:\n",
    "                    ax.imshow(stimuli)\n",
    "                    ax.set_title(\"Filter #{}\".format(filter_index))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-09T22:11:24.717347Z",
     "start_time": "2017-09-09T22:11:24.652428Z"
    }
   },
   "outputs": [],
   "source": [
    "# maximum stimuli for convolutional neurons\n",
    "conv_activation_layers = []\n",
    "for layer in model.layers:\n",
    "    if isinstance(layer, LeakyReLU):\n",
    "        prev_layer = layer.inbound_nodes[0].inbound_layers[0]\n",
    "        if isinstance(prev_layer, Conv2D):\n",
    "            conv_activation_layers.append(layer)\n",
    "\n",
    "for layer in conv_activation_layers:\n",
    "    print(layer.name)\n",
    "    plot_filters_stimuli(layer_name=layer.name, is_conv=True, model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# maximum stimuli for last dense layer\n",
    "last_dense_layer = list(filter(lambda x: isinstance(x, Dense), model.layers))[-1]\n",
    "plot_filters_stimuli(layer_name=last_dense_layer.name, is_conv=False, \n",
    "                     iterations=200, step=0.1, model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def maximum_stimuli_test_for_grader():\n",
    "    layer = list(filter(lambda x: isinstance(x, Dense), model.layers))[-1]\n",
    "    output_index = 7\n",
    "    stimuli, loss = find_maximum_stimuli(\n",
    "        layer_name=layer.name, \n",
    "        is_conv=False, \n",
    "        filter_index=output_index,\n",
    "        model=model,\n",
    "        verbose=False\n",
    "    )\n",
    "    return model.predict_proba(stimuli[np.newaxis, :])[0, output_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-09T22:26:46.387162Z",
     "start_time": "2017-09-09T22:26:46.376735Z"
    }
   },
   "outputs": [],
   "source": [
    "## GRADED PART, DO NOT CHANGE!\n",
    "# Maximum stimuli test\n",
    "grader.set_answer(\"96eco\", maximum_stimuli_test_for_grader())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you can make submission with answers so far to check yourself at this stage\n",
    "grader.submit(COURSERA_EMAIL, COURSERA_TOKEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That's it! Congratulations!\n",
    "\n",
    "What you've done:\n",
    "- defined CNN architecture\n",
    "- trained your model\n",
    "- evaluated your model\n",
    "- visualised learnt filters"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "66px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
